﻿# -*- coding: utf-8 -*-
import re
from urllib import urlopen
from BeautifulSoup import BeautifulSoup

class Reader:

	def __init__(self,url):
		self.url = url
    	
	def remove_html_tags(self,data):
		# remove the newlines
		data = data.replace("\n", " ")
		data = data.replace("\r", " ")
	
		# replace consecutive spaces into a single one
		data = " ".join(data.split())
	
		# now remove the java script
		p = re.compile(r'< script[^<>]*?>.*?< / script >')
		data = p.sub('', data)
   
		# remove the css styles
		p = re.compile(r'< style[^<>]*?>.*?< / style >')
		data = p.sub('', data)
	
		# remove html comments
		p = re.compile(r'')
		data = p.sub('', data)

		# remove all the tags
		p = re.compile(r'<[^<]*?>')
		data = p.sub('', data)
	
		return  data
    
	def remove_extra_spaces(self,data):
		p = re.compile(r'\s+')
		return p.sub(' ', data)
    
	def getParagrapsTabs(self,data):
		soup = BeautifulSoup(data)
		paragraphs = soup.findAll('p')
		str=''
		for paragraph in paragraphs:
			str = str+unicode(paragraph)
		return str
	
	def removeScripts(self,data):
		soup = BeautifulSoup(data)
		[s.extract() for s in soup('script')]
		return unicode(soup)
	
	def removeConstants(self,data):
		data = data.replace('&nbsp;','')
		data = data.replace('&quot;','')
		data = data.replace('-',' ')
		return data.replace('&gt;','')
		

	def getText(self):
		input = unicode(urlopen(self.url).read(), errors='ignore')
		
		data = self.getParagrapsTabs(input)
		data = self.removeScripts(data)
		data = self.removeConstants(data)
		data = self.remove_html_tags(data)
		data = self.remove_extra_spaces(data)
		return unicode(data)